{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Understanding Tree SHAP for Simple Models\n",
    "\n",
    "The SHAP value for a feature is the average change in model output by conditioning on that feature when introducing features one at a time over all feature orderings. While this is easy to state, it is challenging to compute. So this notebook is meant to give a few simple examples where we can see how this plays out for very small trees. For arbitrary large trees it is very hard to intuitively guess these values by looking at the tree."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import graphviz\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.tree import DecisionTreeRegressor, export_graphviz\n",
    "\n",
    "import shap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single split example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"338pt\" height=\"161pt\"\n",
       " viewBox=\"0.00 0.00 338.00 161.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 157)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-157 334,-157 334,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M235.5,-153C235.5,-153 94.5,-153 94.5,-153 88.5,-153 82.5,-147 82.5,-141 82.5,-141 82.5,-101 82.5,-101 82.5,-95 88.5,-89 94.5,-89 94.5,-89 235.5,-89 235.5,-89 241.5,-89 247.5,-95 247.5,-101 247.5,-101 247.5,-141 247.5,-141 247.5,-147 241.5,-153 235.5,-153\"/>\n",
       "<text text-anchor=\"start\" x=\"134.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"143.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"151.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"90.5\" y=\"-124.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"112\" y=\"-110.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"124\" y=\"-96.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M144,-53C144,-53 12,-53 12,-53 6,-53 0,-47 0,-41 0,-41 0,-12 0,-12 0,-6 6,0 12,0 12,0 144,0 144,0 150,0 156,-6 156,-12 156,-12 156,-41 156,-41 156,-47 150,-53 144,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"29.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"37\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M135.79,-88.94C127.23,-79.84 117.83,-69.85 109.16,-60.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"111.47,-57.98 102.07,-53.09 106.37,-62.77 111.47,-57.98\"/>\n",
       "<text text-anchor=\"middle\" x=\"101.3\" y=\"-74.38\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M318,-53C318,-53 186,-53 186,-53 180,-53 174,-47 174,-41 174,-41 174,-12 174,-12 174,-6 180,0 186,0 186,0 318,0 318,0 324,0 330,-6 330,-12 330,-12 330,-41 330,-41 330,-47 324,-53 318,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"182\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"203.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"211\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M194.21,-88.94C202.77,-79.84 212.17,-69.85 220.84,-60.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.63,-62.77 227.93,-53.09 218.53,-57.98 223.63,-62.77\"/>\n",
       "<text text-anchor=\"middle\" x=\"228.7\" y=\"-74.38\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x7fd4dbe5be20>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: N // 2, 0] = 1\n",
    "y[: N // 2] = 1\n",
    "\n",
    "# fit model\n",
    "single_split_model = DecisionTreeRegressor(max_depth=1)\n",
    "single_split_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(\n",
    "    single_split_model,\n",
    "    out_file=None,\n",
    "    filled=True,\n",
    "    rounded=True,\n",
    "    special_characters=True,\n",
    ")\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explaining the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.5). The SHAP value for features not used in the model is always 0, while for $x_0$ it is just the difference between the expected value and the output of the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        x1   x2   x3   x4\n",
       "Example 0 x            1.0  1.0  1.0  1.0\n",
       "          shap_values  0.5  0.0  0.0  0.0\n",
       "Example 1 x            0.0  0.0  0.0  0.0\n",
       "          shap_values -0.5  0.0  0.0  0.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = [np.ones(M), np.zeros(M)]\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(single_split_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x1\", \"x2\", \"x3\", \"x4\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two features AND example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use two features in this example. If feature $x_{0} = 1$ AND $x_{1} = 1$, the target value is one, else zero. Hence we call this the AND model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"430pt\" height=\"261pt\"\n",
       " viewBox=\"0.00 0.00 430.00 261.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 257)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-257 426,-257 426,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#f8e0ce\" stroke=\"black\" d=\"M242,-253C242,-253 92,-253 92,-253 86,-253 80,-247 80,-241 80,-241 80,-201 80,-201 80,-195 86,-189 92,-189 92,-189 242,-189 242,-189 248,-189 254,-195 254,-201 254,-201 254,-241 254,-241 254,-247 248,-253 242,-253\"/>\n",
       "<text text-anchor=\"start\" x=\"136.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"145.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"153.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"88\" y=\"-224.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.188</text>\n",
       "<text text-anchor=\"start\" x=\"114\" y=\"-210.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"121.5\" y=\"-196.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.25</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M144,-147.5C144,-147.5 12,-147.5 12,-147.5 6,-147.5 0,-141.5 0,-135.5 0,-135.5 0,-106.5 0,-106.5 0,-100.5 6,-94.5 12,-94.5 12,-94.5 144,-94.5 144,-94.5 150,-94.5 156,-100.5 156,-106.5 156,-106.5 156,-135.5 156,-135.5 156,-141.5 150,-147.5 144,-147.5\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-132.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"29.5\" y=\"-117.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"37\" y=\"-102.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M138.84,-188.99C129.09,-178.25 118.12,-166.17 108.21,-155.26\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"110.52,-152.61 101.21,-147.56 105.34,-157.31 110.52,-152.61\"/>\n",
       "<text text-anchor=\"middle\" x=\"100.01\" y=\"-168.83\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M327.5,-153C327.5,-153 186.5,-153 186.5,-153 180.5,-153 174.5,-147 174.5,-141 174.5,-141 174.5,-101 174.5,-101 174.5,-95 180.5,-89 186.5,-89 186.5,-89 327.5,-89 327.5,-89 333.5,-89 339.5,-95 339.5,-101 339.5,-101 339.5,-141 339.5,-141 339.5,-147 333.5,-153 327.5,-153\"/>\n",
       "<text text-anchor=\"start\" x=\"226.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"235.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"243.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"182.5\" y=\"-124.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"208.5\" y=\"-110.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"216\" y=\"-96.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M195.48,-188.99C203.77,-179.97 212.92,-169.99 221.59,-160.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"224.21,-162.88 228.4,-153.14 219.06,-158.14 224.21,-162.88\"/>\n",
       "<text text-anchor=\"middle\" x=\"229.46\" y=\"-174.42\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M236,-53C236,-53 104,-53 104,-53 98,-53 92,-47 92,-41 92,-41 92,-12 92,-12 92,-6 98,0 104,0 104,0 236,0 236,0 242,0 248,-6 248,-12 248,-12 248,-41 248,-41 248,-47 242,-53 236,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"100\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"121.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"129\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M227.79,-88.94C219.23,-79.84 209.83,-69.85 201.16,-60.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"203.47,-57.98 194.07,-53.09 198.37,-62.77 203.47,-57.98\"/>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M410,-53C410,-53 278,-53 278,-53 272,-53 266,-47 266,-41 266,-41 266,-12 266,-12 266,-6 272,0 278,0 278,0 410,0 410,0 416,0 422,-6 422,-12 422,-12 422,-41 422,-41 422,-47 416,-53 410,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"274\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"295.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"303\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>2&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M286.21,-88.94C294.77,-79.84 304.17,-69.85 312.84,-60.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.63,-62.77 319.93,-53.09 310.53,-57.98 315.63,-62.77\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x7fd4dbe923e0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: 1 * N // 4, 1] = 1\n",
    "X[: N // 2, 0] = 1\n",
    "X[N // 2 : 3 * N // 4, 1] = 1\n",
    "y[: 1 * N // 4] = 1\n",
    "\n",
    "# fit model\n",
    "and_model = DecisionTreeRegressor(max_depth=2)\n",
    "and_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(and_model, out_file=None, filled=True, rounded=True, special_characters=True)\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explaining the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.25). The SHAP values for the unused features $x_2$ and $x_3$ are always 0. For $x_0$ and $x_1$ it is just the difference between the expected value (0.25) and the output of the model split equally between them (since they equally contribute to the AND function)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>0.375</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.125</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          x1     x2   x3   x4\n",
       "Example 0 x            1.000  1.000  1.0  1.0\n",
       "          shap_values  0.375  0.375  0.0  0.0\n",
       "Example 1 x            0.000  0.000  0.0  0.0\n",
       "          shap_values -0.125 -0.125  0.0  0.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.array([np.ones(M), np.zeros(M)])\n",
    "# np.array([np.ones(M), np.zeros(M), np.array([1, 0, 1, 0]), np.array([0, 1, 0, 0])]   # you can also check these examples\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(and_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x1\", \"x2\", \"x3\", \"x4\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is how you get to the Shap values of Example 1: <br>\n",
    "The bias term (`y.mean()`) is 0.25, and the target value is 1. This leaves 1 - 0.27 = 0.75 to split among the relevant features. Since only $x_1$ and $x_2$ contribute to the target value (and to the same extent), it is divided among them, i.e., 0.375 for each."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two features OR example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We do a slight variation of the example above. If $x_{0} = 1$ OR $x_{1} = 1$ the target is 1, else 0. Can you guess the SHAP values without scrolling down?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"430pt\" height=\"261pt\"\n",
       " viewBox=\"0.00 0.00 430.00 261.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 257)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-257 426,-257 426,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#eca06a\" stroke=\"black\" d=\"M329,-253C329,-253 179,-253 179,-253 173,-253 167,-247 167,-241 167,-241 167,-201 167,-201 167,-195 173,-189 179,-189 179,-189 329,-189 329,-189 335,-189 341,-195 341,-201 341,-201 341,-241 341,-241 341,-247 335,-253 329,-253\"/>\n",
       "<text text-anchor=\"start\" x=\"223.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"232.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"240.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"175\" y=\"-224.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.188</text>\n",
       "<text text-anchor=\"start\" x=\"201\" y=\"-210.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"208.5\" y=\"-196.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.75</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M235.5,-153C235.5,-153 94.5,-153 94.5,-153 88.5,-153 82.5,-147 82.5,-141 82.5,-141 82.5,-101 82.5,-101 82.5,-95 88.5,-89 94.5,-89 94.5,-89 235.5,-89 235.5,-89 241.5,-89 247.5,-95 247.5,-101 247.5,-101 247.5,-141 247.5,-141 247.5,-147 241.5,-153 235.5,-153\"/>\n",
       "<text text-anchor=\"start\" x=\"134.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"143.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"151.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"90.5\" y=\"-124.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"116.5\" y=\"-110.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"124\" y=\"-96.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M225.84,-188.99C217.64,-179.97 208.59,-169.99 200.02,-160.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"202.6,-158.19 193.28,-153.14 197.41,-162.9 202.6,-158.19\"/>\n",
       "<text text-anchor=\"middle\" x=\"192.08\" y=\"-174.41\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M410,-147.5C410,-147.5 278,-147.5 278,-147.5 272,-147.5 266,-141.5 266,-135.5 266,-135.5 266,-106.5 266,-106.5 266,-100.5 272,-94.5 278,-94.5 278,-94.5 410,-94.5 410,-94.5 416,-94.5 422,-100.5 422,-106.5 422,-106.5 422,-135.5 422,-135.5 422,-141.5 416,-147.5 410,-147.5\"/>\n",
       "<text text-anchor=\"start\" x=\"274\" y=\"-132.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"295.5\" y=\"-117.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"303\" y=\"-102.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>0&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M282.48,-188.99C292.44,-178.15 303.65,-165.94 313.75,-154.94\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"316.34,-157.29 320.53,-147.56 311.19,-152.55 316.34,-157.29\"/>\n",
       "<text text-anchor=\"middle\" x=\"321.59\" y=\"-168.83\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M144,-53C144,-53 12,-53 12,-53 6,-53 0,-47 0,-41 0,-41 0,-12 0,-12 0,-6 6,0 12,0 12,0 144,0 144,0 150,0 156,-6 156,-12 156,-12 156,-41 156,-41 156,-47 150,-53 144,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"29.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"37\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M135.79,-88.94C127.23,-79.84 117.83,-69.85 109.16,-60.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"111.47,-57.98 102.07,-53.09 106.37,-62.77 111.47,-57.98\"/>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M318,-53C318,-53 186,-53 186,-53 180,-53 174,-47 174,-41 174,-41 174,-12 174,-12 174,-6 180,0 186,0 186,0 318,0 318,0 324,0 330,-6 330,-12 330,-12 330,-41 330,-41 330,-47 324,-53 318,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"182\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"203.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"211\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M194.21,-88.94C202.77,-79.84 212.17,-69.85 220.84,-60.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.63,-62.77 227.93,-53.09 218.53,-57.98 223.63,-62.77\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x7fd4dbe5bfd0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: N // 2, 0] = 1\n",
    "X[: 1 * N // 4, 1] = 1\n",
    "X[N // 2 : 3 * N // 4, 1] = 1\n",
    "y[: N // 2] = 1\n",
    "y[N // 2 : 3 * N // 4] = 1\n",
    "\n",
    "# fit model\n",
    "or_model = DecisionTreeRegressor(max_depth=2)\n",
    "or_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(or_model, out_file=None, filled=True, rounded=True, special_characters=True)\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explaining the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.75). The SHAP value for features not used in the model is always 0, while for $x_0$ and $x_1$ it is just the difference between the expected value and the output of the model split equally between them (since they equally contribute to the OR function)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>0.125</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.375</td>\n",
       "      <td>-0.375</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          x1     x2   x3   x4\n",
       "Example 0 x            1.000  1.000  1.0  1.0\n",
       "          shap_values  0.125  0.125  0.0  0.0\n",
       "Example 1 x            0.000  0.000  0.0  0.0\n",
       "          shap_values -0.375 -0.375  0.0  0.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.array([np.ones(M), np.zeros(M)])\n",
    "# np.array([np.ones(M), np.zeros(M), np.array([1, 0, 1, 0]), np.array([0, 1, 0, 0])]   # you can also check these examples\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(or_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x1\", \"x2\", \"x3\", \"x4\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two feature XOR example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"690pt\" height=\"261pt\"\n",
       " viewBox=\"0.00 0.00 690.00 261.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 257)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-257 686,-257 686,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M410.5,-253C410.5,-253 269.5,-253 269.5,-253 263.5,-253 257.5,-247 257.5,-241 257.5,-241 257.5,-201 257.5,-201 257.5,-195 263.5,-189 269.5,-189 269.5,-189 410.5,-189 410.5,-189 416.5,-189 422.5,-195 422.5,-201 422.5,-201 422.5,-241 422.5,-241 422.5,-247 416.5,-253 410.5,-253\"/>\n",
       "<text text-anchor=\"start\" x=\"309.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"318.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"326.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"265.5\" y=\"-224.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"287\" y=\"-210.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"299\" y=\"-196.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M319.5,-153C319.5,-153 178.5,-153 178.5,-153 172.5,-153 166.5,-147 166.5,-141 166.5,-141 166.5,-101 166.5,-101 166.5,-95 172.5,-89 178.5,-89 178.5,-89 319.5,-89 319.5,-89 325.5,-89 331.5,-95 331.5,-101 331.5,-101 331.5,-141 331.5,-141 331.5,-147 325.5,-153 319.5,-153\"/>\n",
       "<text text-anchor=\"start\" x=\"218.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"227.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"235.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"174.5\" y=\"-124.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"200.5\" y=\"-110.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"208\" y=\"-96.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M311.21,-188.99C302.83,-179.97 293.57,-169.99 284.81,-160.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"287.29,-158.09 277.92,-153.14 282.16,-162.85 287.29,-158.09\"/>\n",
       "<text text-anchor=\"middle\" x=\"276.99\" y=\"-174.43\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M502.5,-153C502.5,-153 361.5,-153 361.5,-153 355.5,-153 349.5,-147 349.5,-141 349.5,-141 349.5,-101 349.5,-101 349.5,-95 355.5,-89 361.5,-89 361.5,-89 502.5,-89 502.5,-89 508.5,-89 514.5,-95 514.5,-101 514.5,-101 514.5,-141 514.5,-141 514.5,-147 508.5,-153 502.5,-153\"/>\n",
       "<text text-anchor=\"start\" x=\"401.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"410.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"418.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"357.5\" y=\"-124.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"383.5\" y=\"-110.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"391\" y=\"-96.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>0&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M369.11,-188.99C377.58,-179.97 386.95,-169.99 395.8,-160.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"398.47,-162.83 402.76,-153.14 393.37,-158.04 398.47,-162.83\"/>\n",
       "<text text-anchor=\"middle\" x=\"403.55\" y=\"-174.43\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M144,-53C144,-53 12,-53 12,-53 6,-53 0,-47 0,-41 0,-41 0,-12 0,-12 0,-6 6,0 12,0 12,0 144,0 144,0 150,0 156,-6 156,-12 156,-12 156,-41 156,-41 156,-47 150,-53 144,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"29.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"37\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M191.58,-88.94C173.07,-78.93 152.59,-67.85 134.17,-57.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"135.77,-54.77 125.31,-53.09 132.44,-60.93 135.77,-54.77\"/>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M318,-53C318,-53 186,-53 186,-53 180,-53 174,-47 174,-41 174,-41 174,-12 174,-12 174,-6 180,0 186,0 186,0 318,0 318,0 324,0 330,-6 330,-12 330,-12 330,-41 330,-41 330,-47 324,-53 318,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"182\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"203.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"211\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M250.01,-88.94C250.28,-80.66 250.57,-71.64 250.84,-63.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"254.34,-63.2 251.17,-53.09 247.35,-62.97 254.34,-63.2\"/>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>5</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M496,-53C496,-53 364,-53 364,-53 358,-53 352,-47 352,-41 352,-41 352,-12 352,-12 352,-6 358,0 364,0 364,0 496,0 496,0 502,0 508,-6 508,-12 508,-12 508,-41 508,-41 508,-47 502,-53 496,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"360\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"381.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"389\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M431.33,-88.94C431.15,-80.66 430.95,-71.64 430.77,-63.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"434.27,-63.01 430.55,-53.09 427.27,-63.16 434.27,-63.01\"/>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>6</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M670,-53C670,-53 538,-53 538,-53 532,-53 526,-47 526,-41 526,-41 526,-12 526,-12 526,-6 532,0 538,0 538,0 670,0 670,0 676,0 682,-6 682,-12 682,-12 682,-41 682,-41 682,-47 676,-53 670,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"534\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"555.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"563\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;6 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M489.75,-88.94C508.37,-78.93 528.98,-67.85 547.5,-57.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"549.27,-60.91 556.41,-53.09 545.95,-54.74 549.27,-60.91\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x7fd4dbe93f10>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: N // 2, 0] = 1\n",
    "X[: 1 * N // 4, 1] = 1\n",
    "X[N // 2 : 3 * N // 4, 1] = 1\n",
    "y[1 * N // 4 : N // 2] = 1\n",
    "y[N // 2 : 3 * N // 4] = 1\n",
    "\n",
    "# fit model\n",
    "xor_model = DecisionTreeRegressor(max_depth=2)\n",
    "xor_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(xor_model, out_file=None, filled=True, rounded=True, special_characters=True)\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explaining the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.5). The SHAP value for features not used in the model is always 0, while for $x_0$ and $x_1$ it is just the difference between the expected value and the output of the model split equally between them (since they equally contribute to the XOR function)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         x1    x2   x3   x4\n",
       "Example 0 x            1.00  1.00  1.0  1.0\n",
       "          shap_values -0.25 -0.25  0.0  0.0\n",
       "Example 1 x            0.00  0.00  0.0  0.0\n",
       "          shap_values -0.25 -0.25  0.0  0.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.array([np.ones(M), np.zeros(M)])\n",
    "# np.array([np.ones(M), np.zeros(M), np.array([1, 0, 1, 0]), np.array([0, 1, 0, 0])]   # you can also check these examples\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(xor_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x1\", \"x2\", \"x3\", \"x4\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two features AND + feature boost example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"430pt\" height=\"261pt\"\n",
       " viewBox=\"0.00 0.00 430.00 261.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 257)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-257 426,-257 426,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#f5d0b5\" stroke=\"black\" d=\"M242,-253C242,-253 92,-253 92,-253 86,-253 80,-247 80,-241 80,-241 80,-201 80,-201 80,-195 86,-189 92,-189 92,-189 242,-189 242,-189 248,-189 254,-195 254,-201 254,-201 254,-241 254,-241 254,-247 248,-253 242,-253\"/>\n",
       "<text text-anchor=\"start\" x=\"136.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"145.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"153.5\" y=\"-238.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"88\" y=\"-224.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.688</text>\n",
       "<text text-anchor=\"start\" x=\"114\" y=\"-210.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"121.5\" y=\"-196.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.75</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M144,-147.5C144,-147.5 12,-147.5 12,-147.5 6,-147.5 0,-141.5 0,-135.5 0,-135.5 0,-106.5 0,-106.5 0,-100.5 6,-94.5 12,-94.5 12,-94.5 144,-94.5 144,-94.5 150,-94.5 156,-100.5 156,-106.5 156,-106.5 156,-135.5 156,-135.5 156,-141.5 150,-147.5 144,-147.5\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-132.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"29.5\" y=\"-117.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"37\" y=\"-102.3\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M138.84,-188.99C129.09,-178.25 118.12,-166.17 108.21,-155.26\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"110.52,-152.61 101.21,-147.56 105.34,-157.31 110.52,-152.61\"/>\n",
       "<text text-anchor=\"middle\" x=\"100.01\" y=\"-168.83\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#eca06a\" stroke=\"black\" d=\"M327.5,-153C327.5,-153 186.5,-153 186.5,-153 180.5,-153 174.5,-147 174.5,-141 174.5,-141 174.5,-101 174.5,-101 174.5,-95 180.5,-89 186.5,-89 186.5,-89 327.5,-89 327.5,-89 333.5,-89 339.5,-95 339.5,-101 339.5,-101 339.5,-141 339.5,-141 339.5,-147 333.5,-153 327.5,-153\"/>\n",
       "<text text-anchor=\"start\" x=\"226.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"235.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"243.5\" y=\"-138.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"182.5\" y=\"-124.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"208.5\" y=\"-110.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"216\" y=\"-96.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M195.48,-188.99C203.77,-179.97 212.92,-169.99 221.59,-160.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"224.21,-162.88 228.4,-153.14 219.06,-158.14 224.21,-162.88\"/>\n",
       "<text text-anchor=\"middle\" x=\"229.46\" y=\"-174.42\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M236,-53C236,-53 104,-53 104,-53 98,-53 92,-47 92,-41 92,-41 92,-12 92,-12 92,-6 98,0 104,0 104,0 236,0 236,0 242,0 248,-6 248,-12 248,-12 248,-41 248,-41 248,-47 242,-53 236,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"100\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"121.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"129\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M227.79,-88.94C219.23,-79.84 209.83,-69.85 201.16,-60.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"203.47,-57.98 194.07,-53.09 198.37,-62.77 203.47,-57.98\"/>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M410,-53C410,-53 278,-53 278,-53 272,-53 266,-47 266,-41 266,-41 266,-12 266,-12 266,-6 272,0 278,0 278,0 410,0 410,0 416,0 422,-6 422,-12 422,-12 422,-41 422,-41 422,-47 416,-53 410,-53\"/>\n",
       "<text text-anchor=\"start\" x=\"274\" y=\"-37.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"295.5\" y=\"-22.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"303\" y=\"-7.8\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 2.0</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>2&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M286.21,-88.94C294.77,-79.84 304.17,-69.85 312.84,-60.63\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"315.63,-62.77 319.93,-53.09 310.53,-57.98 315.63,-62.77\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x7fd4dbe5bf70>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: N // 2, 0] = 1\n",
    "X[: 1 * N // 4, 1] = 1\n",
    "X[N // 2 : 3 * N // 4, 1] = 1\n",
    "y[: 1 * N // 4] = 1\n",
    "y[: N // 2] += 1\n",
    "\n",
    "# fit model\n",
    "and_fb_model = DecisionTreeRegressor(max_depth=2)\n",
    "and_fb_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(and_fb_model, out_file=None, filled=True, rounded=True, special_characters=True)\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explain the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.75). The SHAP value for features not used in the model is always 0, while for $x_0$ and $x_1$ it is just the difference between the expected value and the output of the model split equally between them (since they equally contribute to the AND function), plus an extra 0.5 impact for $x_0$ since it has an effect of $1.0$ all by itself (+0.5 if it is on and -0.5 if it is off)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>0.875</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.625</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          x1     x2   x3   x4\n",
       "Example 0 x            1.000  1.000  1.0  1.0\n",
       "          shap_values  0.875  0.375  0.0  0.0\n",
       "Example 1 x            0.000  0.000  0.0  0.0\n",
       "          shap_values -0.625 -0.125  0.0  0.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.array([np.ones(M), np.zeros(M)])\n",
    "# np.array([np.ones(M), np.zeros(M), np.array([1, 0, 1, 0]), np.array([0, 1, 0, 0])]   # you can also check these examples\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(and_fb_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x1\", \"x2\", \"x3\", \"x4\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
